package edu.hawaii.webspider;

import java.io.EOFException;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;
import com.meterware.httpunit.GetMethodWebRequest;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebLink;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;
import com.meterware.httpunit.HttpUnitOptions;

/**
 * A demonstration class illustrating how to retrieve and process web pages using HttpUnit.
 * 
 * @author Michal Zielinski
 */
public class WebSpiderExample extends Thread {
  /** Contains the starting url for the crawl. */
  protected final String startUrl;
  private int numberOfLinks;
  private int numberOfPages = 0;
  private final List<String> list = new ArrayList<String>();
  private final List<String> popularList = new ArrayList<String>();
  private final Logger logger = Logger.getLogger("edu.hawaii.webspider");
  private static final String logging = "-logging";
  
  /**
   * format of the date displayer for logging.
   */
  public static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
  
  /**
   * A WebSpider crawls the web and returns info.
   * 
   * @param startUrl The home url for the crawl.
   */
  public WebSpiderExample(String startUrl) {
    this.startUrl = startUrl;
  }

  /**
   * Returns the number of links found at the startUrl.
   * 
   * @return The number of links.
   * @throws Exception if problems occur retrieving the startUrl.
   */
  public int getNumLinks() throws Exception {
    // create the conversation object which will maintain state for us
    WebConversation wc = new WebConversation();
    WebRequest request = new GetMethodWebRequest(this.startUrl);
    WebResponse response = wc.getResponse(request);
    return response.getLinks().length;
  }
  
  /**
   * Prints error messages.
   * 
   */
  public static void printError() {
    System.out.println("EXAMPLE USE 1: java -jar webspider-michalz.jar");
    System.out.println("EXAMPLE USE 2: java -jar webspider-michalz.jar " +
                       "-totallinks http://hackystat.org 10");
    System.out.println("EXAMPLE USE 3: java -jar webspider-michalz.jar " +
    "-mostpopular http://hackystat.org 20 -logging");
  }

  /**
   * Calculates the count and the URL of the most popular page.
   * @param arg if logging 
   * @return string containing the most popular URL and the count
   */
  public String mostPopularPages(String arg) {
    String toReturn;
    int tempPopularCount = 0;
    int countPopular = 0;
    String tempPopularUrl = null;
    String popularUrl = null;

    for (int i = 0; i < popularList.toArray().length; i++) {
      for (int j = i + 1; j < popularList.toArray().length; j++) {
        if (popularList.toArray()[i].equals(popularList.toArray()[j])) {
          if (logging.equals(arg)) {
            logger.info("saving " + popularList.toArray()[i]); 
          }
          tempPopularCount++;
          tempPopularUrl = popularList.toArray()[i].toString();
        }
      }
      if (countPopular < tempPopularCount) {
        countPopular = tempPopularCount;
        popularUrl = tempPopularUrl;
      }
      tempPopularCount = 0;
      tempPopularUrl = null;
    }
    if (countPopular == 0) {
      toReturn = "There is no most popular page";
    }
    else {
      countPopular++;
      toReturn = "Most popular page is " + popularUrl + " with " 
      + countPopular + " links";
    }
    return toReturn; 
  }
  
  /**
   * Follows the links until maximum pages are visited.
   * @param str the URL
   * @param maxPages the maximum pages to follow
   * @param arg if logging
   * @throws Exception If problems occur.
   * @return numberOfLinks found on specified number of pages
   */
  public int traversePages(String str, int maxPages, String arg) throws Exception {
    HttpUnitOptions.setExceptionsThrownOnScriptError(false);
    HttpUnitOptions.setScriptingEnabled(false);
    WebConversation wc = new WebConversation();
    WebRequest request = new GetMethodWebRequest(str);
    WebResponse response = null;
    response = wc.getResponse(request);
    WebLink[] links = response.getLinks();
   
    WebResponse currentPage = wc.getCurrentPage();
    
    if (logging.equals(arg)) {
      int copyOfNumberOfPages = numberOfPages;
      logger.info("retrieving Page# " + ++copyOfNumberOfPages + ": " +
                  currentPage.getURL().toString() + " ,found " 
                  + currentPage.getLinks().length + " links"); 
      }
 
    if (numberOfPages >= 0 && numberOfPages < maxPages) {
      boolean isDifferent = true;
      int count = 0;
      
      for (int i = 0; i < list.toArray().length; i++) {
        if (list.toArray()[i].toString().equals(currentPage.getURL().toString())) {
          popularList.add(list.toArray()[i].toString());
          isDifferent = false;
          count = 1;
        }
        if (logging.equals(arg) && count == 1) {
          logger.info("This page was already visited");
        }
        count = 0;
      }
      
      if (isDifferent) {
        list.add(currentPage.getURL().toString());
        numberOfPages++;
        numberOfLinks = numberOfLinks + links.length;
        
        for (int j = 0; j < links.length; j++) {
          try {
            links[j].click();
          }       
          catch (Exception e) {
            logger.warning("Connection exception caught");
          }
          currentPage = wc.getCurrentPage();
          if (currentPage.isHTML() && numberOfPages < maxPages) {
            try {
            traversePages(currentPage.getURL().toString(), maxPages, arg);
            }
            catch (EOFException e) {
              logger.warning("EOFexception caught");
            }
            catch (Exception e) {
              logger.warning("exception caught");
            }
          }
          else {          
            return numberOfLinks;
          }
        }
      }
    }
    return numberOfLinks;
  }
      
  /**
   * Retrieves the httpunit home page and counts the number of links on it.
   * 
   * @param args Ignored.
   * @throws Exception If problems occur.
   */
  public static void main(String[] args) throws Exception {
    int totalLinks = 0;
    
    if (args.length >= 3) {
      if (args[0].equals("-totallinks") && (args[1].startsWith("http", 0)) 
                                        && (!(args[2].isEmpty()))) {
        boolean isNumeric = true;
        for (int i = 0; i < args[2].length(); i++) {
          Character c = args[2].charAt(i);
          if (!(Character.isDigit(c))) {
            isNumeric = false;
          }
        }
        
        if (isNumeric) {
          WebSpiderExample crawl = new WebSpiderExample(args[1]);  
          
          if (args.length == 3) {
           totalLinks = crawl.traversePages(args[1], Integer.parseInt(args[2]), "nothing");
          }
          else if ("-logging".equals(args[3])) {
           totalLinks = crawl.traversePages(args[1], Integer.parseInt(args[2]), args[3]);
          }
          else {
            printError();
          }
          System.out.println();
          System.out.println("The number of links found = " + totalLinks);
        }
        else {
          printError();
        }
      }
      
      else if (args[0].equals("-mostpopular")) {
        String  mostPopularUrl = null;
        WebSpiderExample crawl = new WebSpiderExample(args[1]);  
        
        if (args.length == 3) {
          totalLinks = crawl.traversePages(args[1], Integer.parseInt(args[2]), "nothing");
          mostPopularUrl = crawl.mostPopularPages("nothing");
        }
        else if ("-logging".equals(args[3])) {
          totalLinks = crawl.traversePages(args[1], Integer.parseInt(args[2]), args[3]);
          mostPopularUrl = crawl.mostPopularPages(args[3]);
        }
        else {
          printError();
        }
        System.out.println();
        System.out.println(mostPopularUrl);
      }
      else {
        printError();
      }
    }
    
    else if (args.length == 0) {
      WebSpiderExample example = new WebSpiderExample("http://www.httpunit.org");
      System.out.println("The HttpUnit home page contains " + example.getNumLinks() + " links.");
    }
    else {
      printError();
    }
  }
}









